{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 基本模块\n",
    "import pandas as pd\n",
    "from requests_html import HTMLSession\n",
    "from urllib.parse import urlparse, parse_qs\n",
    "from IPython.display import display, HTML\n",
    "import time\n",
    "from random import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 测试 为单页抓取做准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<Element 'div' class=('row', 'body-container')>]"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = \"https://news.un.org/zh/search/印度\"\n",
    "session = HTMLSession() #session 模拟人 保有cookie\n",
    "r = session.get( url )\n",
    "主要元素 = r.html.xpath('//div/div[@class=\"row body-container\"]')\n",
    "主要元素"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 尝试抓取元素"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'联合国：全球近1400万婴儿未接种疫苗   新冠疫情或将使接种率进一步下降': '/zh/story/2020/07/1062211',\n",
       " '联合国野生物种犯罪报告：大象犀牛偷猎减少  穿山甲鳞片缴获量五年增加十倍': '/zh/story/2020/07/1061841',\n",
       " '联合国：新冠疫情使': '/zh/story/2020/07/1061142',\n",
       " '联合国人权专家敦促': '/zh/story/2020/06/1060802',\n",
       " '肯尼亚赢得安理会本年度选举的最后一个席位': '/zh/story/2020/06/1060222',\n",
       " '【专题报道】面对种族歧视问题  联合国必须“站在历史正确的一边”': '/zh/story/2020/06/1060142',\n",
       " '【专题报道】屯粮潮后食品价格不升反降 —— 粮农组织高级官员解读亚太地区的粮食安全与卫生状况 ': '/zh/story/2020/06/1059522',\n",
       " '《专题报道》巾帼不让须眉 中国维和女军人受联合国表彰——专访联黎部队高级排雷参谋辛源': '/zh/story/2020/06/1059422',\n",
       " '联合国人权专家：': '/zh/story/2020/06/1059022',\n",
       " '联合国人权高专对中国等亚太国家在疫情期间压制言论自由表示震惊 ': '/zh/story/2020/06/1058802'}"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "标题超链接 = r.html.xpath('//div[contains(@class,\"view-content\")]//section[@class=\"story-block clearfix\"]/h1/a/@href')\n",
    "                                                                  # 只取超链接\n",
    "标题本身 = r.html.xpath('//div[contains(@class,\"view-content\")]//section[@class=\"story-block clearfix\"]/h1/a')\n",
    "标题本身 = { x.xpath(\"a/text()\")[0]:x.xpath(\"a/@href\")[0] for x in 标题本身}\n",
    "标题本身"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<Element 'span' class=('date-display-single',) property='dc:date' datatype='xsd:dateTime' content='2020-07-15T00:00:00-04:00'>,\n",
       " <Element 'span' class=('date-display-single',) property='dc:date' datatype='xsd:dateTime' content='2020-07-10T00:00:00-04:00'>,\n",
       " <Element 'span' class=('date-display-single',) property='dc:date' datatype='xsd:dateTime' content='2020-07-01T00:00:00-04:00'>,\n",
       " <Element 'span' class=('date-display-single',) property='dc:date' datatype='xsd:dateTime' content='2020-06-26T00:00:00-04:00'>,\n",
       " <Element 'span' class=('date-display-single',) property='dc:date' datatype='xsd:dateTime' content='2020-06-18T00:00:00-04:00'>,\n",
       " <Element 'span' class=('date-display-single',) property='dc:date' datatype='xsd:dateTime' content='2020-06-18T00:00:00-04:00'>,\n",
       " <Element 'span' class=('date-display-single',) property='dc:date' datatype='xsd:dateTime' content='2020-06-11T00:00:00-04:00'>,\n",
       " <Element 'span' class=('date-display-single',) property='dc:date' datatype='xsd:dateTime' content='2020-06-10T00:00:00-04:00'>,\n",
       " <Element 'span' class=('date-display-single',) property='dc:date' datatype='xsd:dateTime' content='2020-06-04T00:00:00-04:00'>,\n",
       " <Element 'span' class=('date-display-single',) property='dc:date' datatype='xsd:dateTime' content='2020-06-03T00:00:00-04:00'>]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "时间本身 = r.html.xpath('//div[contains(@class,\"view-content\")]//section[@class=\"story-block clearfix\"]//span[@class=\"date-display-single\"]')                                                                  \n",
    "时间本身"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'健康及卫生': '/zh/news/topic/health',\n",
       " '法律与预防犯罪': '/zh/news/topic/law-and-crime-prevention',\n",
       " '移民与难民': '/zh/news/topic/migrants-and-refugees',\n",
       " '人权': '/zh/news/topic/human-rights',\n",
       " '和平与安全': '/zh/news/topic/peace-and-security',\n",
       " '经济发展': '/zh/news/topic/economic-development'}"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "标签超链接 = r.html.xpath('//div[contains(@class,\"view-content\")]//section[@class=\"story-block clearfix\"]/div[@class=\"topics\"]/a/@href')\n",
    "标签本身 = r.html.xpath('//div[contains(@class,\"view-content\")]//section[@class=\"story-block clearfix\"]/div[@class=\"topics\"]/a')\n",
    "标签本身 = { x.xpath(\"a/text()\")[0]:x.xpath(\"a/@href\")[0] for x in 标签本身}\n",
    "标签本身"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 单一页面抓取 测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>新闻标题</th>\n",
       "      <th>日期</th>\n",
       "      <th>议题分类</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>联合国：全球近1400万婴儿未接种疫苗 新冠疫情或将使接种率进一步下降,联合国野生物种犯罪报...</td>\n",
       "      <td>2020 年 7 月 15 日,2020 年 7 月 10 日,2020 年 7 月 1 日...</td>\n",
       "      <td>健康及卫生,法律与预防犯罪,移民与难民,人权,和平与安全,人权,经济发展,和平与安全,人权,人权</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                新闻标题  \\\n",
       "0  联合国：全球近1400万婴儿未接种疫苗 新冠疫情或将使接种率进一步下降,联合国野生物种犯罪报...   \n",
       "\n",
       "                                                  日期  \\\n",
       "0  2020 年 7 月 15 日,2020 年 7 月 10 日,2020 年 7 月 1 日...   \n",
       "\n",
       "                                               议题分类  \n",
       "0  健康及卫生,法律与预防犯罪,移民与难民,人权,和平与安全,人权,经济发展,和平与安全,人权,人权  "
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "url = \"https://news.un.org/zh/search/印度\"\n",
    "session = HTMLSession() #session 模拟人 保有cookie\n",
    "r = session.get( url )\n",
    "#主要元素 = r.html.xpath('//div[@class=\"view view-search view-id-search view-display-id-search_results_page search-results-page unnews-listings-page teaser view-dom-id-0f84539806d335394b8a0d55da774353\"]')\n",
    "\n",
    "#主要元素 = r.html.xpath('/html/body/div[4]/div[2]')\n",
    "主要元素 = r.html.xpath('//div/div[@class=\"row body-container\"]')\n",
    "主要元素\n",
    "\n",
    "\n",
    "# 作为xpath字典，键为我要抓的牛肉名称，值为xpath\n",
    "dict_xpaths={ \n",
    "    'text': {\n",
    "        '新闻标题':      '//div[contains(@class,\"view-content\")]//section[@class=\"story-block clearfix\"]/h1',\n",
    "        '日期':      '//div[contains(@class,\"view-content\")]//section[@class=\"story-block clearfix\"]//div[@class=\"story-date pull-left\"]',\n",
    "        '议题分类':    '//div[contains(@class,\"view-content\")]//section[@class=\"story-block clearfix\"]/div[@class=\"topics\"]', \n",
    "            }}\n",
    "\n",
    "def get_e_text(_xpath_):\n",
    "    # 高级列表推导\n",
    "    暂存结果 = [\",\".join([x.strip() if type(x) is str else x.text.strip() for x in e.xpath(_xpath_)]) for e in 主要元素]\n",
    "    return(暂存结果)\n",
    "\n",
    "\n",
    "# 只对主要元素下进行.xpath取值\n",
    "数据字典 = dict()\n",
    "\n",
    "\n",
    "\n",
    "数据字典 = {k:get_e_text(v) for k,v in dict_xpaths['text'].items()}\n",
    "# 循环挖取\n",
    "数据字典.update({k:get_e_text(v) for k,v in dict_xpaths['text'].items()})\n",
    "\n",
    "\n",
    "[len(v) for k,v in 数据字典.items()]\n",
    "\n",
    "数据 = pd.DataFrame(数据字典)\n",
    "#数据.to_excel(\"尝试1.xlsx\", sheet_name=\"搜查结果\")\n",
    "数据 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 用split（）对爬取的数据进行进一步的拆分处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "新闻标题 = [ x.split(\",\") for x in 数据字典['新闻标题']]\n",
    "日期标题 = [ x.split(\",\") for x in 数据字典['日期']]\n",
    "议题分类标题 = [ x.split(\",\") for x in 数据字典['议题分类']]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "新闻标题数据 = pd.DataFrame(新闻标题,index=['新闻标题']).T\n",
    "日期标题数据 = pd.DataFrame(日期标题,index=['日期']).T\n",
    "议题分类标题数据 = pd.DataFrame(议题分类标题,index=['议题分类']).T\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>新闻标题</th>\n",
       "      <th>日期</th>\n",
       "      <th>议题分类</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>联合国：全球近1400万婴儿未接种疫苗 新冠疫情或将使接种率进一步下降</td>\n",
       "      <td>2020 年 7 月 15 日</td>\n",
       "      <td>健康及卫生</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>联合国野生物种犯罪报告：大象犀牛偷猎减少 穿山甲鳞片缴获量五年增加十倍</td>\n",
       "      <td>2020 年 7 月 10 日</td>\n",
       "      <td>法律与预防犯罪</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>联合国：新冠疫情使印度境内四万多名难民陷入生存困境</td>\n",
       "      <td>2020 年 7 月 1 日</td>\n",
       "      <td>移民与难民</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>联合国人权专家敦促印度释放示威领袖</td>\n",
       "      <td>2020 年 6 月 26 日</td>\n",
       "      <td>人权</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>肯尼亚赢得安理会本年度选举的最后一个席位</td>\n",
       "      <td>2020 年 6 月 18 日</td>\n",
       "      <td>和平与安全</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>【专题报道】面对种族歧视问题 联合国必须“站在历史正确的一边”</td>\n",
       "      <td>2020 年 6 月 18 日</td>\n",
       "      <td>人权</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>【专题报道】屯粮潮后食品价格不升反降 —— 粮农组织高级官员解读亚太地区的粮食安全与卫生状况</td>\n",
       "      <td>2020 年 6 月 11 日</td>\n",
       "      <td>经济发展</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>《专题报道》巾帼不让须眉 中国维和女军人受联合国表彰——专访联黎部队高级排雷参谋辛源</td>\n",
       "      <td>2020 年 6 月 10 日</td>\n",
       "      <td>和平与安全</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>联合国人权专家：印度必须遵循最高法院的命令 保护1亿“农民工”</td>\n",
       "      <td>2020 年 6 月 4 日</td>\n",
       "      <td>人权</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>联合国人权高专对中国等亚太国家在疫情期间压制言论自由表示震惊</td>\n",
       "      <td>2020 年 6 月 3 日</td>\n",
       "      <td>人权</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             新闻标题               日期     议题分类\n",
       "0             联合国：全球近1400万婴儿未接种疫苗 新冠疫情或将使接种率进一步下降  2020 年 7 月 15 日    健康及卫生\n",
       "1             联合国野生物种犯罪报告：大象犀牛偷猎减少 穿山甲鳞片缴获量五年增加十倍  2020 年 7 月 10 日  法律与预防犯罪\n",
       "2                       联合国：新冠疫情使印度境内四万多名难民陷入生存困境   2020 年 7 月 1 日    移民与难民\n",
       "3                               联合国人权专家敦促印度释放示威领袖  2020 年 6 月 26 日       人权\n",
       "4                            肯尼亚赢得安理会本年度选举的最后一个席位  2020 年 6 月 18 日    和平与安全\n",
       "5                 【专题报道】面对种族歧视问题 联合国必须“站在历史正确的一边”  2020 年 6 月 18 日       人权\n",
       "6  【专题报道】屯粮潮后食品价格不升反降 —— 粮农组织高级官员解读亚太地区的粮食安全与卫生状况  2020 年 6 月 11 日     经济发展\n",
       "7      《专题报道》巾帼不让须眉 中国维和女军人受联合国表彰——专访联黎部队高级排雷参谋辛源  2020 年 6 月 10 日    和平与安全\n",
       "8                 联合国人权专家：印度必须遵循最高法院的命令 保护1亿“农民工”   2020 年 6 月 4 日       人权\n",
       "9                  联合国人权高专对中国等亚太国家在疫情期间压制言论自由表示震惊   2020 年 6 月 3 日       人权"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = [新闻标题数据, 日期标题数据, 议题分类标题数据]\n",
    "数据表 = pd.concat(df,axis=1)\n",
    "数据表"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 建构参数模板：找到关键参数及参数结构"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element 'a' title='Go to page 2' href='/zh/search/%E5%8D%B0%E5%BA%A6?page=1'>, <Element 'a' title='Go to page 3' href='/zh/search/%E5%8D%B0%E5%BA%A6?page=2'>, <Element 'a' title='Go to page 4' href='/zh/search/%E5%8D%B0%E5%BA%A6?page=3'>, <Element 'a' title='Go to page 5' href='/zh/search/%E5%8D%B0%E5%BA%A6?page=4'>, <Element 'a' title='Go to next page' href='/zh/search/%E5%8D%B0%E5%BA%A6?page=1'>, <Element 'a' title='Go to last page' href='/zh/search/%E5%8D%B0%E5%BA%A6?page=46'>]\n",
      "{'2': '/zh/search/%E5%8D%B0%E5%BA%A6?page=1', '3': '/zh/search/%E5%8D%B0%E5%BA%A6?page=2', '4': '/zh/search/%E5%8D%B0%E5%BA%A6?page=3', '5': '/zh/search/%E5%8D%B0%E5%BA%A6?page=4', '下一页': '/zh/search/%E5%8D%B0%E5%BA%A6?page=1', '尾页 »': '/zh/search/%E5%8D%B0%E5%BA%A6?page=46'}\n"
     ]
    }
   ],
   "source": [
    "#  xpath 解析翻页a/@href\n",
    "\n",
    "xpath_翻页a = '//div[contains(@class,\"text-center\")]/ul/li/a'\n",
    "print (r.html.xpath(xpath_翻页a)) # 物件\n",
    "\n",
    "href_列表 = [x.xpath('//@href')[0] for x in r.html.xpath(xpath_翻页a)]\n",
    "#print (href_列表)\n",
    "\n",
    "文字_列表 = [x.text for x in r.html.xpath(xpath_翻页a)]\n",
    "#print (文字_列表)\n",
    "\n",
    "href_字典 = {x.text:x.xpath('//@href')[0]  for x in r.html.xpath(xpath_翻页a)}\n",
    "print (href_字典)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### urlparse 解析后丢入数据框"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scheme</th>\n",
       "      <th>netloc</th>\n",
       "      <th>path</th>\n",
       "      <th>params</th>\n",
       "      <th>query</th>\n",
       "      <th>fragment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zh/search/%E5%8D%B0%E5%BA%A6</td>\n",
       "      <td></td>\n",
       "      <td>page=1</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zh/search/%E5%8D%B0%E5%BA%A6</td>\n",
       "      <td></td>\n",
       "      <td>page=2</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zh/search/%E5%8D%B0%E5%BA%A6</td>\n",
       "      <td></td>\n",
       "      <td>page=3</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zh/search/%E5%8D%B0%E5%BA%A6</td>\n",
       "      <td></td>\n",
       "      <td>page=4</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zh/search/%E5%8D%B0%E5%BA%A6</td>\n",
       "      <td></td>\n",
       "      <td>page=1</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zh/search/%E5%8D%B0%E5%BA%A6</td>\n",
       "      <td></td>\n",
       "      <td>page=46</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  scheme netloc                           path params    query fragment\n",
       "0                /zh/search/%E5%8D%B0%E5%BA%A6          page=1         \n",
       "1                /zh/search/%E5%8D%B0%E5%BA%A6          page=2         \n",
       "2                /zh/search/%E5%8D%B0%E5%BA%A6          page=3         \n",
       "3                /zh/search/%E5%8D%B0%E5%BA%A6          page=4         \n",
       "4                /zh/search/%E5%8D%B0%E5%BA%A6          page=1         \n",
       "5                /zh/search/%E5%8D%B0%E5%BA%A6         page=46         "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scheme      1\n",
      "netloc      1\n",
      "path        1\n",
      "params      1\n",
      "query       5\n",
      "fragment    1\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>page</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  page\n",
       "0    1\n",
       "1    2\n",
       "2    3\n",
       "3    4\n",
       "4    1\n",
       "5   46"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "page    5\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "\n",
    "df = pd.DataFrame([ urlparse(x) for x in href_列表])\n",
    "df_qs = pd.DataFrame([{k:v[0] for k,v in parse_qs(x).items()} for x in df['query'] ])\n",
    "\n",
    "display(df)\n",
    "print(df.nunique())\n",
    "display(df_qs)\n",
    "print(df_qs.nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>page</th>\n",
       "      <th>page_int</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  page  page_int\n",
       "0    1         1\n",
       "1    2         2\n",
       "2    3         3\n",
       "3    4         4\n",
       "4    1         1\n",
       "5   46        46"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_qs.page\n",
    "df_qs = df_qs.assign (page_int=df_qs.page.astype(int)) # 变成整数\n",
    "df_qs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 取一例做模板"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'page': ['1']}\n",
      "{'2': '/zh/search/%E5%8D%B0%E5%BA%A6?page=1', '3': '/zh/search/%E5%8D%B0%E5%BA%A6?page=2', '4': '/zh/search/%E5%8D%B0%E5%BA%A6?page=3', '5': '/zh/search/%E5%8D%B0%E5%BA%A6?page=4', '下一页': '/zh/search/%E5%8D%B0%E5%BA%A6?page=1', '尾页 »': '/zh/search/%E5%8D%B0%E5%BA%A6?page=46'}\n"
     ]
    }
   ],
   "source": [
    "def parse_url_qs_for_page (url):\n",
    "    six_parts = urlparse(url) \n",
    "    out = parse_qs(six_parts.query)\n",
    "    return (out)\n",
    "\n",
    "参数模板 = parse_url_qs_for_page(href_列表[0])\n",
    "print (参数模板)\n",
    "\n",
    "print (href_字典)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 建构参数模板生成器：page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "46\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def 参数模板生成( page):\n",
    "    参数 = 参数模板.copy()\n",
    "    参数['page'] = page\n",
    "    \n",
    "    return (参数)\n",
    "\n",
    "参数_翻页_page = { \n",
    "    i:参数模板生成(page = [i]) \\\n",
    "    for i in href_字典.items()\\\n",
    "     }\n",
    "\n",
    "# print(参数_keyword_用户体验_curPage) # 只生成本页有的额外翻页URL\n",
    "\n",
    "print (df_qs.page_int.min()) # 最小值为1\n",
    "print (df_qs.page_int.max()) # 最大值为46\n",
    "\n",
    "# 应该是 0 (本页)....46(最大值)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {'page': [0]},\n",
       " 1: {'page': [1]},\n",
       " 2: {'page': [2]},\n",
       " 3: {'page': [3]},\n",
       " 4: {'page': [4]},\n",
       " 5: {'page': [5]},\n",
       " 6: {'page': [6]},\n",
       " 7: {'page': [7]},\n",
       " 8: {'page': [8]},\n",
       " 9: {'page': [9]},\n",
       " 10: {'page': [10]},\n",
       " 11: {'page': [11]},\n",
       " 12: {'page': [12]},\n",
       " 13: {'page': [13]},\n",
       " 14: {'page': [14]},\n",
       " 15: {'page': [15]},\n",
       " 16: {'page': [16]},\n",
       " 17: {'page': [17]},\n",
       " 18: {'page': [18]},\n",
       " 19: {'page': [19]},\n",
       " 20: {'page': [20]},\n",
       " 21: {'page': [21]},\n",
       " 22: {'page': [22]},\n",
       " 23: {'page': [23]},\n",
       " 24: {'page': [24]},\n",
       " 25: {'page': [25]},\n",
       " 26: {'page': [26]},\n",
       " 27: {'page': [27]},\n",
       " 28: {'page': [28]},\n",
       " 29: {'page': [29]},\n",
       " 30: {'page': [30]},\n",
       " 31: {'page': [31]},\n",
       " 32: {'page': [32]},\n",
       " 33: {'page': [33]},\n",
       " 34: {'page': [34]},\n",
       " 35: {'page': [35]},\n",
       " 36: {'page': [36]},\n",
       " 37: {'page': [37]},\n",
       " 38: {'page': [38]},\n",
       " 39: {'page': [39]},\n",
       " 40: {'page': [40]},\n",
       " 41: {'page': [41]},\n",
       " 42: {'page': [42]},\n",
       " 43: {'page': [43]},\n",
       " 44: {'page': [44]},\n",
       " 45: {'page': [45]},\n",
       " 46: {'page': [46]}}"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "参数_翻页_page = { \n",
    "    i:参数模板生成(page = [i]) \\\n",
    "    for i in range(0,df_qs.page_int.max()+1)\\\n",
    "    }\n",
    "参数_翻页_page"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 翻页进行时"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "#   单一页面\n",
    "url = \"https://news.un.org/zh/search/印度\"\n",
    "session = HTMLSession() #session 模拟人 保有cookie\n",
    "\n",
    "def 印度( url, params):\n",
    "    r = session.get( url , params= payload)\n",
    "    主要元素 = r.html.xpath('//div/div[@class=\"row body-container\"]')\n",
    "     \n",
    "    \n",
    "    dict_xpaths={ \n",
    "    'text': {\n",
    "        '新闻标题':      '//div[contains(@class,\"view-content\")]//section[@class=\"story-block clearfix\"]/h1',\n",
    "        '日期':      '//div[contains(@class,\"view-content\")]//section[@class=\"story-block clearfix\"]//div[@class=\"story-date pull-left\"]',\n",
    "        '议题分类':    '//div[contains(@class,\"view-content\")]//section[@class=\"story-block clearfix\"]/div[@class=\"topics\"]', \n",
    "            }}\n",
    "\n",
    "    def get_e_text(_xpath_):\n",
    "        # 高级列表推导\n",
    "        暂存结果 = [\",\".join([x.strip() if type(x) is str else x.text.strip() for x in e.xpath(_xpath_)]) for e in 主要元素]\n",
    "        return(暂存结果)\n",
    "\n",
    "\n",
    "    # 只对主要元素下进行.xpath取值\n",
    "    数据字典 = dict()\n",
    "\n",
    "    数据字典 = {k:get_e_text(v) for k,v in dict_xpaths['text'].items()}\n",
    "    数据字典.update({k:get_e_text(v) for k,v in dict_xpaths['text'].items()})\n",
    "\n",
    "    新闻标题 = [ x.split(\",\") for x in 数据字典['新闻标题']]\n",
    "    日期标题 = [ x.split(\",\") for x in 数据字典['日期']]\n",
    "    议题分类标题 = [ x.split(\",\") for x in 数据字典['议题分类']]\n",
    "    \n",
    "    \n",
    "    \n",
    "    新闻标题数据 = pd.DataFrame(新闻标题,index=['新闻标题']).T\n",
    "    日期标题数据 = pd.DataFrame(日期标题,index=['日期']).T\n",
    "    议题分类标题数据 = pd.DataFrame(议题分类标题,index=['议题分类']).T\n",
    "    \n",
    "    df = [新闻标题数据, 日期标题数据, 议题分类标题数据]\n",
    "    数据表 = pd.concat(df,axis=1)\n",
    "\n",
    "    \n",
    "    \n",
    "    数据 = pd.DataFrame(数据表)\n",
    "    #数据.to_excel(\"翻页14尝试.xlsx\", sheet_name=\"搜查结果\")\n",
    "    return (数据)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "list_df = list()\n",
    "for k,v in 参数_翻页_page.items():\n",
    "    payload = v\n",
    "    df = 印度( url, params = payload)\n",
    "\n",
    "    time.sleep(1+4*random())  #放慢脚步 3-7秒, 平均约5秒\n",
    "    df = df.assign (page = k)  # 区分  page\n",
    "    list_df.append(df)\n",
    "\n",
    "df_all = pd.concat(list_df)\n",
    "#df_all.index.name = '序'\n",
    "df_all    \n",
    "df_all.to_excel(\"Indian_news.xlsx\", sheet_name=\"搜查结果\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
